package com._21cn.fetch.utils;

import com._21cn.cms.article.entity.Article;
import com._21cn.cms.article.utils.FormatContentUtil;
import com._21cn.framework.http.HttpCaller;
import com._21cn.framework.utils.DateUtil;
import com._21cn.framework.utils.StringUtil;
import com.alibaba.fastjson.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.print.Doc;
import java.io.*;
import java.util.*;
import java.util.regex.Pattern;

/**
 * Created by Administrator on 2019/3/4.
 */
public class Fetch21CN {

    private static String postUrl = "http://admin.cms.qiaoliba.com/admin/article/article/save.do";

    private static Pattern urlPattern = Pattern.compile("/(\\d{6,9})\\.shtml");

    public static Article getArticleContent(String contentUrl){
        Document doc = getDocument(contentUrl);
        Article article = formatArticle(doc);
        String title = doc.select(".title").text();
        String prefix = getPrefix(contentUrl);
        Set<String> pager = getPager(doc);
        for (String pageUrl : pager) {
            Document document = getDocument(prefix+pageUrl);
            String content = FormatContentUtil.formatContent(document.select("#article_text").html());
            if (!StringUtil.isEmpty(content)) {
                content = article.getContent() + "<p class=\"pager\">&nbsp;</p>" + content;
                article.setContent(content);
            }
        }
        article.setTitle(title);
        return article;
    }

    public static String getPrefix(String contentUrl){
        String result = contentUrl;
        if(!StringUtil.isEmpty(contentUrl)){
            contentUrl = contentUrl.substring(contentUrl.length()-14, contentUrl.length());
            result = result.replace(contentUrl, "");
        }
        return result;
    }

    public static Article formatArticle(Document doc) {
        Article article = new Article();
        String title = doc.select(".title").text();
        String content = FormatContentUtil.formatContent(doc.select("#article_text").html());
        article.setTitle(title);
        article.setContent(content);
        return article;
    }

    public static Document getDocument(String url){
        HttpCaller httpCaller = HttpCaller.get(url);
        httpCaller.addHeader("Referer","21cn.com");
        httpCaller.addHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3514.0 Safari/537.36");
        String html = httpCaller.call().getString();
        Document doc = Jsoup.parse(html);
        doc.select("#embed_hzh_div").remove();
        return doc;
    }

    private static String linkPattern = "\\d+_\\d+.shtml";
    private static Set<String> linkSet = new TreeSet();

    public static Set<String> getPager(Document document)
    {
        Elements itemElements = document.select(".page-box .page-ctrl").select("a");
        for (Element element : itemElements) {
            String link = element.attr("href");
            if (Pattern.matches(linkPattern, link)) {
                linkSet.add(link);
            }
        }
        return linkSet;
    }

    public static Map<String, String> extractPictureListFromContent(String content) {
        String fetUrl = "http://admin.cms.qiaoliba.com/pictureFetch.do?status=1";
        Map resultMap = new HashMap();
        try {
            String sourceUrl;
            JSONObject array;
            if (!StringUtil.isEmpty(content)) {
                Document doc = Jsoup.parse(content);
                Elements imgs = doc.getElementsByTag("img");
                int i = 0;
                String urls = "";
                for (Element img : imgs) {
                    sourceUrl = img.attr("src").trim();
                    urls = urls + ",21cn.com," + sourceUrl;
                }
                if (!StringUtil.isEmpty(urls)) {
                    urls = urls.substring(",21cn.com,".length(), urls.length());
                }
                String json = HttpCaller.post(fetUrl).addParam("url", urls).call().getString();
                System.out.println("json:" + json);
                array = JSONObject.parseObject(json);
                for (Element img : imgs) {
                    sourceUrl = img.attr("src").trim();
                    JSONObject obj = array.getJSONObject(sourceUrl);
                    String url = obj.getString("url");
                    resultMap.put(sourceUrl, url);
                }
            }
        }catch (Exception e){
            System.out.println("=========NO picture========");
        }

        return resultMap;
    }

    public static void saveArticle(String articleUrl){
//        Article article = getArticleContent("http://et.21cn.com/movie/xinwen/waiyu/a/2019/0228/11/33162668.shtml");
        Article article = getArticleContent(articleUrl);
        String content = article.getContent();
        Map<String, String> urlmap = extractPictureListFromContent(article.getContent());
        if( urlmap.isEmpty()){
            return;
        }
        for (String url : urlmap.keySet()) {
            if( !StringUtil.isEmpty(urlmap.get(url)) ){
                content = content.replace(url.replaceAll("&","&amp;"), (CharSequence) urlmap.get(url));
            }
        }
        System.out.println(urlmap);

        HttpCaller.post(postUrl).addParam("title", StringUtil.utf82iso(article.getTitle()))
                .addParam("content", StringUtil.utf82iso(content)).addParam("sourceName", StringUtil.utf82iso("21CN娱乐"))
                .addParam("mainColumnId", "37")         // FIXME: 2019/3/5
                .addParam("publishStatus", "2")
                .addParam("topTimeStr", DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss"))
                .addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
                .addHeader("Accept-Encoding", "gzip, deflate")
                .addHeader("Content-Type", "application/x-www-form-urlencoded")
                .addHeader("Cookie", "admin_auth_key=xxt:value_admin")
                .call();
    }


    public static void main(String[] args) {
        BufferedReader br = null;
        try {
            br = new BufferedReader(new InputStreamReader(new FileInputStream("D:/article.txt"), "UTF-8"));
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        }
        String line = "";
        try {
            while ((line = br.readLine()) != null)
                saveArticle(line);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

}
